

def PLex():
	buffStr=""
	TokensList=[]
	plainFile = open("test.txt","r")
	stream = plainFile.read()
	state=0
	print "Lexim"+"\t\t"+"Type"
	for i in stream:
		if state==0:
			if (scanLetter(i) == True):
				buffStr+=i
				state=1
			elif (scanDigits(i) == True):
				buffStr+=i
				state=3
			elif (scanOperators(i) == True):
				buffStr=i
				TokensList.append(buffStr)
				state=0
				buffStr=""
			elif (scanWhiteSpace(i) == True):
				continue
			else:
				buffStr=i
				TokensList.append(buffStr)
				state=0
				buffStr=""
				
		elif state==1:
			if (scanDigits(i) == True) or (scanLetter(i) == True):
				buffStr+=i
			else:
				TokensList.append(buffStr)
				state=0
				buffStr=""
		
		elif state==3:
			if (scanDigits(i) == True):
				buffStr+=i
			elif (i == '.'):
				buffStr+=i
				state=4
			else:
				TokensList.append(buffStr)
				state=0
				buffStr=""
		
		elif state==4:
			if (scanDigits(i) == True):
				buffStr+=i
			else:
				TokensList.append(buffStr)
				state=0
				buffStr=""
		
		elif state==7:
			if (scanOperators == True):
				buffStr+=i
			else:
				TokensList.append(buffStr)
				state=0
				buffStr=""
		
	print "=======>",TokensList
			
			

def scanWhiteSpace(scannedText):
	if (scannedText == " ") or (scannedText == "") or (scannedText == "\n") or (scannedText == "\t"):
		return True
	else:
		return False



def scanLetter(scannedText):
	if (scannedText >= 'A' and scannedText <= 'Z') or (scannedText >= 'a' and scannedText <= 'z') or (scannedText =='_'):
		return True
	else:
		return False



def scanDigits(scannedText):
	if (scannedText >= '0' and scannedText <= '9'):
		return True
	else:
		return False



def scanOperators(scannedText):
	if (scannedText == '+') or (scannedText == '-') or (scannedText =='*'):
		return True
	else:
		return False
		
		
if __name__ == '__main__':
	PLex()
